#!/usr/bin/env python
# encoding: utf-8
"""
freqFilter.py

Created by Pablo Ortega Mesa on 2011-04-17.
Copyright (c) 2011 __MyCompanyName__. All rights reserved.
"""

import sys
import os
import re
import getopt
from operator import itemgetter

class Filter():
	def __init__(self,path,terms):
		self.termsFilename = terms
		self.docs = self.__getDocs(path)
		self.terms = self.__getTerms(terms)
		self.termsFreq = self.calculateFreq()
		
	def __getDocs(self,path):
		filesPossible = os.listdir(path)
		files = []
		for f in filesPossible:
			if re.search('(txt)$',f):
				files.append(path+f)
		print "Listed "+str(len(files))+" docs"
		return files
	
	def __getTerms(self,filename):
		terms = []
		termsFile = open(filename,'r')
		for line in termsFile:
			line = line.split("\n")[0]
			terms.append(line)
		termsFile.close()
		print "Load "+str(len(terms))+" terms"
		return terms
	
	def calculateFreq(self):
		tf = {}
		for term in self.terms:
			cont = 0
			patron = re.compile('.*('+term+').*',re.IGNORECASE)
			for doc in self.docs:
				dataFile = open(doc,'r')
				data = dataFile.read()
				dataFile.close()
				if patron.search(data):
					founds = patron.findall(data)
					cont += len(founds)
			tf[term] = cont
		return tf
	
	def __sort(self):
		return sorted(self.termsFreq.items(), key=itemgetter(1),reverse=True)
	
	def cutList(self,k):
		termsFile = open(self.termsFilename,'w')
		termsSorted = self.__sort()
		cont = 0
		for t in termsSorted:
			cont+=1
			if cont<=k:
				termsFile.write(t[0]+'\n')
			else:
				break
		termsFile.close()
		print "Write top "+str(k)+" terms discriminated by frecuency"
		


options, remainder = getopt.getopt(sys.argv[1:], 'p:t:k:', ['path=', 
		                                                         'terms=',
																 'kcut='])

path = False
terms = False
kcut = 100
for opt, arg in options:
	if opt in ('-p','--path'):
		path = arg
	if opt in ('-t','--terms'):
		terms = arg
	if opt in ('-k','--kcut'):
		kcut = int(arg)

f = Filter(path,terms)
f.cutList(kcut)